# -*- coding: utf-8 -*-

# Scrapy settings for kickstart project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#

BOT_NAME = 'kickstart'

SPIDER_MODULES = ['kickstart.spiders']
NEWSPIDER_MODULE = 'kickstart.spiders'
DOWNLOAD_DELAY = 1
RANDOMIZE_DOWNLOAD_DELAY = True
COOKIES_ENABLES = False
#CONCURRENT_REQUESTS = 100
MONGODB_SERVER = '127.0.0.1'
MONGODB_PORT = 54545 
MONGODB_DB = 'ctrip'
#Tables used in mongodb
#HTML = 'html'
#PIC_TABLE = 'pic'
#CITY = 'city'
#HOTEL = 'hotel'
#COMMENT = 'comment'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'kickstart (+http://www.yourdomain.com)'

DOWNLOADER_MIDDLEWARES = {
 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware' : None,        
 'kickstart.spiders.common_agent.RotateUserAgentMiddleware' :400
 #'kickstart.spiders.common_agent.ProxyMiddleware' :410
 #'kickstart.spiders.randomproxy.RandomProxy' :200

}
LOG_LEVEL = 'DEBUG'
HTTP_PROXY = 'http://127.0.0.1:8123'

ITEM_PIPELINES = {
 'kickstart.pipelines.CtripPipeline': 100
}
# Retry many times since proxies often fail
#RETRY_TIMES = 10
# Retry on most error codes since proxies fail for different reasons
#RETRY_HTTP_CODES = [500, 503, 504, 400, 403, 404, 408]

#PROXY_LIST = '/home/jimmy/kickstart/proxy.list'
